#!/usr/bin/env python
# --*-- conding:utf-8 --*--
# @Time : 2024/1/24
# @Author : liangpp
# @Email : mobaicloud@163.com
# @Software : PyCharm


import os
import csv
import requests
from lxml import etree
import pandas as pd
import django

# os.environ.setdefault("DJANGO_SETTINGS_MODULE",'backend.settings')
# 单独python使用django组件，要设置环境变量并setup
os.environ['DJANGO_SETTINGS_MODULE'] = 'backend.settings'
django.setup()
from carlargescreen.models import CarInfo


class Spider():
    def __init__(self):
        self.url = 'https://www.dongchedi.com/motor/pc/car/rank_data?aid=1839&app_name=auto_web_pc&city_name=%E6%B5%8E%E5%8D%97&month=&new_energy_type=&rank_data_type=11&brand_id=&price=&manufacturer=&outter_detail_type=&nation=0'
        self.header = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36'
        }
        self.params = {  # 控制爬取的数量
            'count': 1,
            'offset': 299
        }
        self.infourl = 'https://www.dongchedi.com/auto/params-carIds-x-'

    def init_csvfile(self):
        if not os.path.exists('./cardata.csv'):
            with open('./cardata.csv', 'a', newline='', encoding='utf-8') as f:
                write = csv.writer(f)
                write.writerow(["brand_name", "car_name", "car_img", "count", "price", "manufacturer", "rank", "model",
                                "energyType", "marketTime", "warrantyPeriod"])

    def insert_to_csv(self, data):
        if type(data) == list:
            try:
                with open('./cardata.csv', 'a', newline='', encoding='utf-8') as f:
                    write = csv.writer(f)
                    write.writerow(data)
            except Exception:
                return 'ERROR: 数据插入csv文件失败'
        else:
            return 'ERROR: data不是列表，请检查'

    def handle_csv(self):  # 使用pandas对csv文件处理
        # df_csv = pd.read_csv('./cardata.csv', sep='\001')
        df_csv = pd.read_csv('./cardata.csv')
        # df_csv.dropna(inplace=True)  # 清除空行
        df_csv.drop_duplicates(inplace=True)  # 清除重复数据
        print(f"总计{df_csv.shape[0]}行数据")
        return df_csv.values

    def save_to_db(self):
        cars_info = self.handle_csv()
        for car in cars_info:
            try:
                CarInfo.objects.create(
                    brand_name=car[0],
                    car_name=car[1],
                    car_img=car[2],
                    count=car[3],
                    price=car[4],
                    manufacturer=car[5],
                    rank=car[6],
                    model=car[7],
                    energyType=car[8],
                    marketTime=car[9],
                    warrantyPeriod=car[10]
                )
            except Exception as e:
                print(e)
        return "save data to DB success!"

    def main(self):
        try:
            jsonresponse = requests.get(self.url, headers=self.header, params=self.params).json()
            car_data = jsonresponse['data']['list']
            for car in car_data:
                car_info = []
                brand_name = car['brand_name']
                car_name = car['series_name']
                car_img = car['image']
                count = car['count']
                manufacturer = car['sub_brand_name']
                rank = car['rank']

                min_price = car['min_price']
                max_privce = car['max_price']
                price = str(min_price) + '-' + str(max_privce)

                # 获取参数页信息
                id = car['series_id']
                info_url = self.infourl + str(id)  # 汽车详细配置页面
                web_HTML = requests.get(info_url, headers=self.header)
                info_HTML = etree.HTML(web_HTML.text)

                # 通过xpath方法获取具体元素位置，元素和列表不同，从1开始 不是从0开始 ，获取后是一个列表，可以通过[0].text获取内容
                try:
                    model = info_HTML.xpath('//div[@data-row-anchor="jb"]/div[2]/div/text()')[0]
                except Exception:
                    model = ""
                try:
                    energyType = info_HTML.xpath('//div[@data-row-anchor="fuel_form"]/div[2]/div/text()')[0]
                except Exception:
                    energyType = ""
                try:
                    marketTime = info_HTML.xpath('//div[@data-row-anchor="market_time"]/div[2]/div/text()')[0]
                except Exception:
                    marketTime = ""
                try:
                    warrantyPeriod = info_HTML.xpath('//div[@data-row-anchor="period"]/div[2]/div/text()')[0]
                except Exception as e:
                    warrantyPeriod = ""

                car_info.extend(
                    [brand_name, car_name, car_img, count, price, manufacturer, rank, model, energyType, marketTime,
                     warrantyPeriod])
                # self.insert_to_csv(data=car_info)
                print(car_info)
        except Exception as e:
            print(e)
            pass
        self.handle_csv()


if __name__ == '__main__':
    spiderman = Spider()
    # spiderman.init_csvfile()
    # spiderman.main()
    # spiderman.save_to_db()
